#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
TPCC 参数化性能测试工具 (Python 2.7) - v2.0
功能：
1. 支持通过配置文件定义多个测试场景
2. 自动化执行不同参数组合的TPCC测试
3. 收集和对比测试结果
4. 生成测试报告
5. 统一的日志输出系统
"""

from __future__ import print_function
import os
import sys
import subprocess
import datetime
import time
import json
import re
import shutil
import csv
from collections import OrderedDict
import logging
import io
import codecs

# ============== 日志系统 ==============
class Logger(object):
    """统一的日志输出系统"""
    
    # 日志级别定义
    LEVELS = {
        'DEBUG': logging.DEBUG,
        'INFO': logging.INFO,
        'WARNING': logging.WARNING,
        'ERROR': logging.ERROR,
        'CRITICAL': logging.CRITICAL
    }
    
    def __init__(self, level='INFO', log_file=None):
        """
        初始化日志系统
        :param level: 日志级别 (DEBUG, INFO, WARNING, ERROR, CRITICAL)
        :param log_file: 日志文件路径，None则只输出到控制台
        """
        self.level = self.LEVELS.get(level.upper(), logging.INFO)
        self.log_file = log_file
        
        # 创建logger
        self.logger = logging.getLogger('TPCC_TEST')
        self.logger.setLevel(self.level)
        
        # 清除已有的handlers
        self.logger.handlers = []
        
        # 设置格式
        formatter = logging.Formatter(
            '%(asctime)s [%(levelname)s] %(message)s',
            datefmt='%Y-%m-%d %H:%M:%S'
        )
        
        # 添加控制台handler
        console_handler = logging.StreamHandler(sys.stdout)
        console_handler.setLevel(self.level)
        console_handler.setFormatter(formatter)
        self.logger.addHandler(console_handler)
        
        # 如果指定了日志文件，添加文件handler
        if self.log_file:
            try:
                # 确保日志目录存在
                log_dir = os.path.dirname(self.log_file)
                if log_dir and not os.path.exists(log_dir):
                    os.makedirs(log_dir)
                
                file_handler = logging.FileHandler(self.log_file, encoding='utf-8')
                file_handler.setLevel(self.level)
                file_handler.setFormatter(formatter)
                self.logger.addHandler(file_handler)
            except Exception as e:
                self.logger.error(u"创建日志文件失败: %s" % str(e))
    
    def _ensure_unicode(self, msg):
        """确保消息是Unicode字符串 (Python 2.7)"""
        if isinstance(msg, unicode):
            return msg
        elif isinstance(msg, str):
            try:
                return msg.decode('utf-8')
            except UnicodeDecodeError:
                # Windows系统可能使用GBK编码
                try:
                    return msg.decode('gbk')
                except:
                    return msg.decode('utf-8', 'ignore')
        else:
            return unicode(msg)
    
    def debug(self, msg):
        """调试信息"""
        self.logger.debug(self._ensure_unicode(msg))
    
    def info(self, msg):
        """一般信息"""
        self.logger.info(self._ensure_unicode(msg))
    
    def warning(self, msg):
        """警告信息"""
        self.logger.warning(self._ensure_unicode(msg))
    
    def error(self, msg):
        """错误信息"""
        self.logger.error(self._ensure_unicode(msg))
    
    def critical(self, msg):
        """严重错误"""
        self.logger.critical(self._ensure_unicode(msg))
    
    def separator(self, char='=', length=70):
        """输出分隔线"""
        self.info(char * length)
    
    def header(self, title, char='=', length=70):
        """输出标题头"""
        self.separator(char, length)
        self.info(title)
        self.separator(char, length)
    
    def section(self, title, level=1):
        """输出章节标题"""
        if level == 1:
            self.info(u"\n=== %s ===" % title)
        elif level == 2:
            self.info(u"\n--- %s ---" % title)
        else:
            self.info(u"\n%s" % title)
    
    def progress(self, current, total, prefix=u"进度"):
        """输出进度信息"""
        # 确保prefix是unicode
        if not isinstance(prefix, unicode):
            if isinstance(prefix, str):
                prefix = prefix.decode('utf-8', 'ignore')
            else:
                prefix = unicode(prefix)
        self.info(u"\n【%s: %d/%d】" % (prefix, current, total))
    
    def result(self, key, value):
        """输出结果键值对"""
        self.info(u"  %s: %s" % (key, value))

# 创建全局logger实例
# 从配置文件或环境变量获取日志配置
LOG_LEVEL = os.environ.get('TPCC_LOG_LEVEL', 'INFO')
LOG_FILE = os.environ.get('TPCC_LOG_FILE', 'logs/tpcc_test.log')
logger = Logger(level=LOG_LEVEL, log_file=LOG_FILE)

# ============== 全局配置参数 ==============
# 路径配置
DM_BIN_DIR = "../dm8/bin"
DM_LOG_DIR = "../dm8/log"
INIT_SCRIPT_DIR = "../init_dm8"
TPCC_LOG_DIR = "log/01_tpcc"
BACKUP_DIR = "config_backup"

# 需要更新配置的文件列表
SCRIPT_FILES = {
    'load': 'tpcc_load.sh',
    'test': 'tpcc_test.sh',
    'init': '../init_dm8/1initDB.sh',
    'dm_ini': '../init_dm8/dm.ini',
    'props_dm': '01_tpcc/tpcc/benchmarksql5/run/props.dm'
}


class ConfigLoader(object):
    """配置文件加载器"""
    
    def __init__(self, config_file='tpcc_test_matrix.json'):
        self.config_file = config_file
        self.config = None
        
    def load(self):
        """加载配置文件"""
        if not os.path.exists(self.config_file):
            raise IOError("配置文件不存在: %s" % self.config_file)
            
        try:
            with io.open(self.config_file, 'r', encoding='utf-8') as f:
                self.config = json.load(f)
            self.validate()
            return self.config
        except ValueError as e:
            raise ValueError("配置文件格式错误: %s" % str(e))
    
    def validate(self):
        """验证配置完整性"""
        # 顶层必需字段
        required_keys = ['database', 'test_scenarios', 'test_config']
        for key in required_keys:
            if key not in self.config:
                raise ValueError("配置文件缺少必要字段: %s" % key)
        
        # 数据库配置必需字段
        required_db_keys = ['host', 'port', 'user', 'password']
        for key in required_db_keys:
            if key not in self.config['database']:
                raise ValueError("database配置缺少必要字段: %s" % key)
        
        # 测试配置必需字段
        required_test_config_keys = ['cleanup_between_tests', 'collect_system_metrics', 
                                     'skip_data_load', 'output_dir', 'log_level', 
                                     'retry_on_failure', 'wait_between_tests']
        for key in required_test_config_keys:
            if key not in self.config['test_config']:
                raise ValueError("test_config缺少必要字段: %s" % key)
        
        # 验证每个测试场景
        for i, scenario in enumerate(self.config['test_scenarios']):
            scenario_name = scenario.get('name', 'scenario_%d' % i)
            
            # 场景必需字段
            required_scenario_keys = ['name', 'description', 'warehouses', 'terminals',
                                     'loadworkers', 'runmins', 'table_type', 
                                     'tbs_size', 'db_params', 'test_flags']
            for key in required_scenario_keys:
                if key not in scenario:
                    raise ValueError("测试场景 %s 缺少必要字段: %s" % (scenario_name, key))
            
            # 如果是分区表，必须有partition字段
            if scenario['table_type'] == 12:
                if 'partition' not in scenario:
                    raise ValueError("测试场景 %s 使用分区表但缺少partition字段" % scenario_name)
            
            # 验证test_flags必需字段
            required_test_flags = ['acid_check', 'checkpoint', 'preheat']
            for key in required_test_flags:
                if key not in scenario['test_flags']:
                    raise ValueError("测试场景 %s 的test_flags缺少必要字段: %s" % (scenario_name, key))
        
        logger.info(u"配置文件验证通过，包含 %d 个测试场景" % len(self.config['test_scenarios']))


class TestScenario(object):
    """测试场景管理"""
    
    def __init__(self, scenario_config, db_config):
        # 所有字段都必需，不设默认值
        self.name = scenario_config['name']
        self.description = scenario_config['description']
        self.warehouses = scenario_config['warehouses']
        self.terminals = scenario_config['terminals']
        self.loadworkers = scenario_config['loadworkers']
        self.runmins = scenario_config['runmins']
        self.table_type = scenario_config['table_type']
        self.tbs_size = scenario_config['tbs_size']
        self.db_params = scenario_config['db_params']
        self.test_flags = scenario_config['test_flags']
        self.db_config = db_config
        
        # 分区字段根据表类型决定
        if self.table_type == 12:
            if 'partition' not in scenario_config:
                raise ValueError("分区表必须指定partition参数")
            self.partition = scenario_config['partition']
        else:
            self.partition = None
        
    def __str__(self):
        return u"场景: %s (仓库=%d, 并发=%d, 运行=%d分钟)" % (
            self.name, self.warehouses, self.terminals, self.runmins)
    
    def get_load_params(self):
        """获取数据加载参数"""
        params = {
            'WAREHOUSES': str(self.warehouses),
            'LOADWORKERS': str(self.loadworkers),
            'TABLE_TYPE': str(self.table_type),
            'TBS_SIZE': str(self.tbs_size),
            'DB_HOST1': self.db_config['host'],
            'DB_PORT1': self.db_config['port'],
            'SYSDBA_PWD': self.db_config['password']
        }
        # 只有分区表才设置PARTITION
        if self.partition is not None:
            params['PARTITION'] = str(self.partition)
        return params
    
    def get_test_params(self):
        """获取性能测试参数"""
        # test_flags必须包含所有必需字段，在验证阶段已检查
        return {
            'WAREHOUSES': str(self.warehouses),
            'TERMINALS': str(self.terminals),
            'RUNMINS': str(self.runmins),
            'DB_HOST1': self.db_config['host'],
            'DB_PORT1': self.db_config['port'],
            'SYSDBA_PWD': self.db_config['password'],
            'ACID_FLAG': '1' if self.test_flags['acid_check'] else '0',
            'CKPT_FLAG': '1' if self.test_flags['checkpoint'] else '0',
            'PREHEAT_FALG': '1' if self.test_flags['preheat'] else '0'
        }


class ScriptUpdater(object):
    """脚本参数更新器"""
    
    def __init__(self):
        self.backup_dir = BACKUP_DIR
        
    def create_backup(self, file_path):
        """备份文件"""
        if not os.path.exists(self.backup_dir):
            os.makedirs(self.backup_dir)
            
        if os.path.exists(file_path):
            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
            backup_name = os.path.basename(file_path) + "." + timestamp
            backup_path = os.path.join(self.backup_dir, backup_name)
            shutil.copy2(file_path, backup_path)
            return backup_path
        return None
    
    def update_shell_variable(self, file_path, var_name, value):
        """更新Shell脚本中的变量"""
        if not os.path.exists(file_path):
            logger.warning(u"文件不存在 - %s" % file_path)
            return False
            
        try:
            with open(file_path, 'r') as f:
                lines = f.readlines()
            
            updated = False
            for i, line in enumerate(lines):
                # 匹配变量赋值行
                if line.strip().startswith('%s=' % var_name):
                    # 保留原有的引号风格
                    if '"' in line:
                        lines[i] = '%s="%s"\n' % (var_name, value)
                    else:
                        lines[i] = '%s=%s\n' % (var_name, value)
                    updated = True
                    break
            
            if updated:
                with codecs.open(file_path, 'w', encoding='utf-8') as f:
                    f.writelines(lines)
                return True
            return False
            
        except Exception as e:
            logger.error(u"更新变量失败: %s - %s" % (var_name, str(e)))
            return False
    
    def update_dm_ini_param(self, ini_path, param_name, value):
        """更新dm.ini参数"""
        if not os.path.exists(ini_path):
            logger.warning(u"dm.ini文件不存在 - %s" % ini_path)
            return False
            
        try:
            with open(ini_path, 'r') as f:
                lines = f.readlines()
            
            found = False
            for i, line in enumerate(lines):
                if line.strip().startswith('%s=' % param_name):
                    lines[i] = '%s=%s\n' % (param_name, value)
                    found = True
                    break
            
            # 如果参数不存在，添加到文件末尾
            if not found:
                if lines and not lines[-1].endswith('\n'):
                    lines[-1] += '\n'
                lines.append('%s=%s\n' % (param_name, value))
            
            with codecs.open(ini_path, 'w', encoding='utf-8') as f:
                f.writelines(lines)
            return True
            
        except Exception as e:
            logger.error(u"更新dm.ini参数失败: %s - %s" % (param_name, str(e)))
            return False
    
    def apply_scenario_config(self, scenario):
        """应用场景配置到脚本文件"""
        logger.info(u"应用配置: %s" % scenario.name)
        
        # 备份配置文件
        for script_key, script_path in SCRIPT_FILES.items():
            if os.path.exists(script_path):
                backup_path = self.create_backup(script_path)
                if backup_path:
                    logger.info(u"  备份: %s" % backup_path)
        
        # 更新tpcc_load.sh
        load_params = scenario.get_load_params()
        for var_name, value in load_params.items():
            self.update_shell_variable(SCRIPT_FILES['load'], var_name, value)
        logger.info(u"  已更新: tpcc_load.sh")
        
        # 更新tpcc_test.sh
        test_params = scenario.get_test_params()
        for var_name, value in test_params.items():
            self.update_shell_variable(SCRIPT_FILES['test'], var_name, value)
        logger.info(u"  已更新: tpcc_test.sh")
        
        # 更新dm.ini参数
        if scenario.db_params:
            for param_name, value in scenario.db_params.items():
                self.update_dm_ini_param(SCRIPT_FILES['dm_ini'], param_name, value)
            logger.info(u"  已更新: dm.ini (%d个参数)" % len(scenario.db_params))
        
        return True


class TestExecutor(object):
    """测试执行器"""
    
    def __init__(self, scenario):
        self.scenario = scenario
        self.log_file = "test_%s.log" % scenario.name
        
    def execute_command(self, command, description=""):
        """执行Shell命令"""
        logger.info(u"执行: %s" % (description or command))
        try:
            # 设置环境变量确保UTF-8编码
            env = os.environ.copy()
            env['PYTHONIOENCODING'] = 'utf-8'
            
            process = subprocess.Popen(
                command,
                shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                env=env
            )
            stdout, stderr = process.communicate()
            
            # 解码输出 (Python 2.7)
            if stdout:
                try:
                    stdout = stdout.decode('utf-8')
                except UnicodeDecodeError:
                    stdout = stdout.decode('gbk', 'ignore')
            else:
                stdout = ''
                
            if stderr:
                try:
                    stderr = stderr.decode('utf-8')
                except UnicodeDecodeError:
                    stderr = stderr.decode('gbk', 'ignore')
            else:
                stderr = ''
            
            # 记录日志 (使用codecs处理编码)
            import codecs
            with codecs.open(self.log_file, 'a', encoding='utf-8') as f:
                timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                f.write(u"\n[%s] %s\n" % (timestamp, description if isinstance(description, unicode) else description.decode('utf-8', 'ignore')))
                f.write(u"命令: %s\n" % (command if isinstance(command, unicode) else command.decode('utf-8', 'ignore')))
                f.write(u"输出:\n%s\n" % stdout)
                if stderr:
                    f.write(u"错误:\n%s\n" % stderr)
            
            return process.returncode, stdout, stderr
        except Exception as e:
            error_msg = unicode(str(e), 'utf-8', 'ignore') if not isinstance(str(e), unicode) else str(e)
            logger.error(u"命令执行失败: %s" % error_msg)
            return -1, '', error_msg
    
    def init_database(self):
        """初始化数据库"""
        logger.section(u"步骤1: 初始化数据库")
        init_script = os.path.join(INIT_SCRIPT_DIR, "1initDB.sh")
        
        if not os.path.exists(init_script):
            logger.warning(u"初始化脚本不存在，跳过初始化")
            return True
            
        returncode, stdout, stderr = self.execute_command(
            "sh %s" % init_script,
            u"数据库初始化"
        )
        
        if returncode == 0:
            logger.info(u"数据库初始化成功")
            return True
        else:
            logger.error(u"数据库初始化失败: %s" % stderr)
            return False
    
    def load_data(self):
        """加载测试数据"""
        logger.section(u"步骤2: 加载测试数据")
        logger.info(u"仓库数: %d, 加载并发: %d" % (self.scenario.warehouses, self.scenario.loadworkers))
        
        returncode, stdout, stderr = self.execute_command(
            "sh %s" % SCRIPT_FILES['load'],
            u"TPCC数据加载"
        )
        
        if returncode == 0:
            logger.info(u"数据加载成功")
            return True
        else:
            logger.error(u"数据加载失败: %s" % stderr)
            return False
    
    def run_test(self):
        """执行性能测试"""
        logger.section(u"步骤3: 执行性能测试")
        logger.info(u"并发数: %d, 运行时长: %d分钟" % (self.scenario.terminals, self.scenario.runmins))
        
        returncode, stdout, stderr = self.execute_command(
            "sh %s" % SCRIPT_FILES['test'],
            u"TPCC性能测试"
        )
        
        if returncode == 0:
            logger.info(u"性能测试完成")
            return True, stdout
        else:
            logger.error(u"性能测试失败: %s" % stderr)
            return False, None
    
    def collect_numa_stats(self):
        """收集NUMA统计信息"""
        logger.info(u"收集NUMA统计信息...")
        try:
            # 获取dmserver进程ID
            cmd = "pidof dmserver"
            returncode, stdout, stderr = self.execute_command(cmd, u"获取dmserver PID")
            
            if returncode == 0 and stdout.strip():
                pid = stdout.strip()
                # 收集NUMA统计
                numa_cmd = "numastat -p %s | sed -n '10p;19p;28p'" % pid
                returncode, stdout, stderr = self.execute_command(numa_cmd, u"NUMA统计")
                
                # 写入test.log
                with codecs.open('test.log', 'a', encoding='utf-8') as f:
                    f.write(u"\n=== NUMA统计信息 ===\n")
                    f.write(stdout if isinstance(stdout, unicode) else stdout.decode('utf-8', 'ignore'))
                    
                return stdout
            else:
                logger.warning(u"未找到dmserver进程")
                return None
        except Exception as e:
            logger.error(u"收集NUMA统计失败: %s" % str(e))
            return None
    
    def collect_db_parameters(self):
        """收集数据库参数"""
        logger.info(u"收集数据库参数...")
        results = {}
        
        try:
            # 获取ENABLE_FREQROOTS参数
            disql_path = os.path.join(DM_BIN_DIR, "disql")
            conn_str = "%s/%s:%s" % (
                self.scenario.db_config['user'],
                self.scenario.db_config['password'],
                self.scenario.db_config['port']
            )
            
            # ENABLE_FREQROOTS查询
            sql_cmd = 'echo -e "ENABLE_FREQROOTS\\n" | %s %s -e "select para_value from v\\$dm_ini where para_name=\'ENABLE_FREQROOTS\'" | tail -n 1' % (
                disql_path, conn_str
            )
            returncode, stdout, stderr = self.execute_command(sql_cmd, u"查询ENABLE_FREQROOTS")
            
            if returncode == 0:
                freqroots_value = stdout.strip()
                results['ENABLE_FREQROOTS'] = freqroots_value
                with codecs.open('test.log', 'a', encoding='utf-8') as f:
                    f.write(u"ENABLE_FREQROOTS: %s\n" % freqroots_value)
            
            # 查询分区表数量
            partition_cmd = '%s %s -e "select count(*) from dba_tab_partitions" | tail -n 1' % (
                disql_path, conn_str
            )
            returncode, stdout, stderr = self.execute_command(partition_cmd, u"查询分区表数量")
            
            if returncode == 0:
                partition_count = stdout.strip()
                results['partitions'] = partition_count
                with codecs.open('test.log', 'a', encoding='utf-8') as f:
                    f.write(u"partitions: %s\n" % partition_count)
                    
        except Exception as e:
            logger.error(u"收集数据库参数失败: %s" % str(e))
            
        return results
    
    def collect_transaction_logs(self):
        """收集事务提交日志"""
        logger.info(u"收集事务日志...")
        try:
            # 获取当前月份的日志文件
            current_month = datetime.datetime.now().strftime("%Y%m")
            log_file = os.path.join(DM_LOG_DIR, "dm_DMSERVER_%s.log" % current_month)
            
            if os.path.exists(log_file):
                # 提取trx4_cmt信息
                cmd = "tail %s | grep trx4_cmt" % log_file
                returncode, stdout, stderr = self.execute_command(cmd, u"提取事务提交信息")
                
                with codecs.open('test.log', 'a', encoding='utf-8') as f:
                    f.write(u"\n=== 事务提交信息 ===\n")
                    if returncode == 0 and stdout:
                        f.write(stdout if isinstance(stdout, unicode) else stdout.decode('utf-8', 'ignore'))
                    else:
                        f.write("未找到事务提交信息\n")
                        
                return stdout
            else:
                logger.warning(u"日志文件不存在 - %s" % log_file)
                return None
        except Exception as e:
            logger.error(u"收集事务日志失败: %s" % str(e))
            return None
    
    def extract_tpcc_results(self):
        """提取TPCC测试结果"""
        logger.info(u"提取TPCC测试结果...")
        try:
            if os.path.exists(TPCC_LOG_DIR):
                # 获取最新的日志文件
                import glob
                log_files = glob.glob(os.path.join(TPCC_LOG_DIR, "*"))
                
                if log_files:
                    # 按修改时间排序，获取最新文件
                    latest_log = max(log_files, key=os.path.getmtime)
                    
                    # 提取tpmC结果
                    cmd = 'grep -A 4 "Measured tpmC (NewOrders)" %s' % latest_log
                    returncode, stdout, stderr = self.execute_command(cmd, u"提取tpmC结果")
                    
                    with codecs.open('test.log', 'a', encoding='utf-8') as f:
                        f.write(u"\n=== TPCC测试结果 ===\n")
                        if returncode == 0 and stdout:
                            f.write(stdout if isinstance(stdout, unicode) else stdout.decode('utf-8', 'ignore'))
                        else:
                            f.write("未找到tpmC结果\n")
                            
                    return stdout
                else:
                    logger.warning(u"未找到TPCC日志文件")
                    return None
            else:
                logger.warning(u"TPCC日志目录不存在")
                return None
        except Exception as e:
            logger.error(u"提取TPCC结果失败: %s" % str(e))
            return None
    
    def execute_scenario(self, skip_load=False):
        """执行完整的测试场景"""
        logger.separator()
        logger.info(u"开始执行测试场景: %s" % self.scenario.name)
        if self.scenario.description:
            logger.info(u"描述: %s" % self.scenario.description)
        logger.separator()
        
        start_time = datetime.datetime.now()
        
        # 初始化数据库
        if not self.init_database():
            return None
        
        # 加载数据（可选）
        if not skip_load:
            if not self.load_data():
                return None
        else:
            logger.info(u"跳过数据加载（使用现有数据）")
        
        # 执行测试
        success, output = self.run_test()
        if not success:
            return None
        
        # === 收集后续性能指标（原始脚本的后续步骤）===
        logger.section(u"步骤4: 收集性能指标")
        
        # 收集各项指标
        numa_stats = self.collect_numa_stats()
        db_params = self.collect_db_parameters()
        transaction_logs = self.collect_transaction_logs()
        tpcc_results = self.extract_tpcc_results()
        
        end_time = datetime.datetime.now()
        duration = (end_time - start_time).total_seconds()
        
        logger.info(u"测试场景完成，耗时: %.1f 秒" % duration)
        
        # 返回包含所有指标的结果
        return {
            'output': output,
            'duration': duration,
            'start_time': start_time.strftime("%Y-%m-%d %H:%M:%S"),
            'end_time': end_time.strftime("%Y-%m-%d %H:%M:%S"),
            'numa_stats': numa_stats,
            'db_params': db_params,
            'transaction_logs': transaction_logs,
            'tpcc_results': tpcc_results
        }


class ResultCollector(object):
    """结果收集器"""
    
    def __init__(self, output_dir=None):
        if output_dir is None:
            raise ValueError("必须指定output_dir参数")
        self.output_dir = output_dir
        self.results = []
        
        if not os.path.exists(self.output_dir):
            os.makedirs(self.output_dir)
    
    def extract_tpmc(self, output):
        """从输出中提取tpmC值"""
        if not output:
            return None
            
        # 查找tpmC相关的行
        patterns = [
            r'Measured tpmC.*?=\s*([\d,]+)',
            r'tpmC:\s*([\d,]+)',
            r'NewOrders.*?tpmC.*?([\d,]+)'
        ]
        
        for pattern in patterns:
            match = re.search(pattern, output, re.IGNORECASE)
            if match:
                # 移除逗号并转换为整数
                tpmc_str = match.group(1).replace(',', '')
                try:
                    return int(tpmc_str)
                except ValueError:
                    pass
        
        return None
    
    def extract_metrics_from_log(self):
        """从日志文件提取更多指标"""
        metrics = {}
        
        # 查找最新的TPCC日志
        if os.path.exists(TPCC_LOG_DIR):
            import glob
            log_files = glob.glob(os.path.join(TPCC_LOG_DIR, "*"))
            if log_files:
                latest_log = max(log_files, key=os.path.getmtime)
                
                try:
                    with open(latest_log, 'r') as f:
                        content = f.read()
                    
                    # 提取各种指标
                    tpmc_match = re.search(r'Measured tpmC.*?=\s*([\d,\.]+)', content)
                    if tpmc_match:
                        metrics['tpmc'] = float(tpmc_match.group(1).replace(',', ''))
                    
                    # 提取响应时间
                    rt_match = re.search(r'Average.*?Response Time.*?(\d+\.?\d*)', content)
                    if rt_match:
                        metrics['avg_response_time'] = float(rt_match.group(1))
                        
                except Exception as e:
                    logger.error(u"读取日志文件失败: %s" % str(e))
        
        return metrics
    
    def collect(self, scenario, test_result):
        """收集测试结果"""
        if not test_result:
            logger.warning(u"测试结果为空，跳过收集")
            return None
        
        # 基本结果
        result = {
            'scenario_name': scenario.name,
            'description': scenario.description,
            'warehouses': scenario.warehouses,
            'terminals': scenario.terminals,
            'loadworkers': scenario.loadworkers,
            'runmins': scenario.runmins,
            'table_type': scenario.table_type,
            'start_time': test_result.get('start_time'),
            'end_time': test_result.get('end_time'),
            'duration_seconds': test_result.get('duration')
        }
        
        # 提取性能指标
        output = test_result.get('output', '')
        result['tpmc'] = self.extract_tpmc(output)
        
        # 从日志文件提取额外指标
        log_metrics = self.extract_metrics_from_log()
        result.update(log_metrics)
        
        # 添加新收集的性能指标
        if 'db_params' in test_result and test_result['db_params']:
            result['enable_freqroots'] = test_result['db_params'].get('ENABLE_FREQROOTS', 'N/A')
            result['partition_count'] = test_result['db_params'].get('partitions', 'N/A')
        
        # 添加NUMA统计信息标记
        result['numa_collected'] = test_result.get('numa_stats') is not None
        
        # 添加事务日志标记
        result['transaction_logs_collected'] = test_result.get('transaction_logs') is not None
        
        # 添加数据库参数
        for param, value in scenario.db_params.items():
            result['db_' + param] = value
        
        self.results.append(result)
        
        logger.section(u"收集到的指标", 2)
        logger.result(u"tpmC", result.get('tpmc', 'N/A'))
        logger.result(u"平均响应时间", result.get('avg_response_time', 'N/A'))
        logger.result(u"ENABLE_FREQROOTS", result.get('enable_freqroots', 'N/A'))
        logger.result(u"分区表数量", result.get('partition_count', 'N/A'))
        logger.result(u"NUMA统计", u'已收集' if result.get('numa_collected') else u'未收集')
        logger.result(u"事务日志", u'已收集' if result.get('transaction_logs_collected') else u'未收集')
        
        return result
    
    def save_csv(self):
        """保存CSV格式的结果"""
        if not self.results:
            logger.info(u"没有结果可保存")
            return
        
        timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
        csv_file = os.path.join(self.output_dir, "results_%s.csv" % timestamp)
        
        # 获取所有字段
        fieldnames = []
        for result in self.results:
            for key in result.keys():
                if key not in fieldnames:
                    fieldnames.append(key)
        
        # 写入CSV
        with codecs.open(csv_file, 'w', encoding='utf-8') as f:
            writer = csv.DictWriter(f, fieldnames=fieldnames)
            writer.writeheader()
            writer.writerows(self.results)
        
        logger.info(u"CSV结果已保存: %s" % csv_file)
        return csv_file
    
    def generate_summary(self):
        """生成汇总报告"""
        if not self.results:
            return
        
        timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
        summary_file = os.path.join(self.output_dir, "summary_%s.md" % timestamp)
        
        with codecs.open(summary_file, 'w', encoding='utf-8') as f:
            f.write(u"# TPCC性能测试报告\n\n")
            f.write(u"生成时间: %s\n\n" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
            
            f.write("## 测试结果汇总\n\n")
            f.write("| 场景名称 | 仓库数 | 并发数 | 运行时长 | tpmC | 平均响应时间 |\n")
            f.write("|---------|--------|--------|----------|------|-------------|\n")
            
            for result in self.results:
                f.write("| %s | %d | %d | %d分钟 | %s | %s |\n" % (
                    result.get('scenario_name', ''),
                    result.get('warehouses', 0),
                    result.get('terminals', 0),
                    result.get('runmins', 0),
                    result.get('tpmc', 'N/A'),
                    result.get('avg_response_time', 'N/A')
                ))
            
            f.write("\n## 详细配置\n\n")
            for result in self.results:
                f.write("### %s\n\n" % result.get('scenario_name', ''))
                f.write("- 描述: %s\n" % result.get('description', ''))
                f.write("- 表类型: %s\n" % result.get('table_type', ''))
                f.write("- 数据库参数:\n")
                for key, value in result.items():
                    if key.startswith('db_'):
                        f.write("  - %s: %s\n" % (key[3:], value))
                f.write("\n")
        
        logger.info(u"汇总报告已生成: %s" % summary_file)
        return summary_file


class TestOrchestrator(object):
    """测试编排器"""
    
    def __init__(self, config_file='tpcc_test_matrix.json'):
        self.config_loader = ConfigLoader(config_file)
        self.config = None
        self.updater = ScriptUpdater()
        self.collector = None  # 将在initialize中创建
        
    def initialize(self):
        """初始化配置"""
        global logger
        
        logger.header(u"TPCC 参数化性能测试工具 v2.0")
        
        logger.info(u"加载配置文件...")
        self.config = self.config_loader.load()
        
        # 设置日志级别
        if 'log_level' in self.config['test_config']:
            log_level = self.config['test_config']['log_level']
            logger = Logger(level=log_level, log_file=LOG_FILE)
        
        # 创建结果收集器，使用配置中的输出目录
        self.collector = ResultCollector(output_dir=self.config['test_config']['output_dir'])
        
        # 显示测试计划
        logger.section(u"测试计划", 2)
        for i, scenario_config in enumerate(self.config['test_scenarios'], 1):
            scenario = TestScenario(scenario_config, self.config['database'])
            logger.info(u"  %d. %s" % (i, scenario))
        
        logger.section(u"测试配置", 2)
        logger.result(u"输出目录", self.config['test_config']['output_dir'])
        logger.result(u"清理数据", self.config['test_config']['cleanup_between_tests'])
        logger.result(u"跳过加载", self.config['test_config']['skip_data_load'])
        
        return True
    
    def run_all_scenarios(self):
        """执行所有测试场景"""
        if not self.initialize():
            return False
        
        total_scenarios = len(self.config['test_scenarios'])
        skip_load = self.config['test_config']['skip_data_load']
        wait_time = self.config['test_config']['wait_between_tests']
        
        logger.separator()
        logger.info(u"开始执行 %d 个测试场景" % total_scenarios)
        logger.separator()
        
        for i, scenario_config in enumerate(self.config['test_scenarios'], 1):
            logger.progress(i, total_scenarios)
            
            # 创建场景对象
            scenario = TestScenario(scenario_config, self.config['database'])
            
            # 应用配置
            if not self.updater.apply_scenario_config(scenario):
                logger.error(u"配置应用失败，跳过场景: %s" % scenario.name)
                continue
            
            # 执行测试
            executor = TestExecutor(scenario)
            test_result = executor.execute_scenario(skip_load=skip_load)
            
            # 收集结果
            if test_result:
                self.collector.collect(scenario, test_result)
            else:
                logger.error(u"测试执行失败: %s" % scenario.name)
            
            # 场景间等待
            if i < total_scenarios and wait_time > 0:
                logger.info(u"等待 %d 秒后执行下一个场景..." % wait_time)
                time.sleep(wait_time)
        
        # 生成报告
        self.generate_reports()
        
        logger.separator()
        logger.info(u"所有测试场景执行完成")
        logger.separator()
        
        return True
    
    def generate_reports(self):
        """生成测试报告"""
        logger.info(u"生成测试报告...")
        
        # 保存CSV结果
        csv_file = self.collector.save_csv()
        
        # 生成Markdown汇总
        summary_file = self.collector.generate_summary()
        
        # 显示结果摘要
        if self.collector.results:
            logger.section(u"测试结果摘要", 2)
            logger.separator('-', 60)
            for result in self.collector.results:
                logger.info(u"%s: tpmC=%s" % (
                    result.get('scenario_name', 'Unknown'),
                    result.get('tpmc', 'N/A')
                ))


def main():
    """主函数"""
    import argparse
    
    parser = argparse.ArgumentParser(
        description='TPCC参数化性能测试工具',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
使用示例:
  python2 %(prog)s                          # 使用默认配置文件 tpcc_test_matrix.json
  python2 %(prog)s my_config.json           # 使用指定配置文件
  python2 %(prog)s my_config.json --check   # 仅检查配置文件
        """
    )
    
    parser.add_argument('config',
                       nargs='?',
                       default='tpcc_test_matrix.json',
                       help='测试配置文件路径 (默认: tpcc_test_matrix.json)')
    
    parser.add_argument('--check',
                       action='store_true',
                       help='仅检查配置文件，不执行测试')
    
    args = parser.parse_args()
    
    try:
        if args.check:
            # 仅验证配置文件
            loader = ConfigLoader(args.config)
            config = loader.load()
            logger.info(u"配置文件检查通过")
            logger.info(u"包含 %d 个测试场景" % len(config['test_scenarios']))
            for scenario in config['test_scenarios']:
                logger.info(u"  - %s" % scenario['name'])
        else:
            # 执行测试
            orchestrator = TestOrchestrator(args.config)
            orchestrator.run_all_scenarios()
            
    except KeyboardInterrupt:
        logger.warning(u"测试被用户中断")
        sys.exit(1)
    except Exception as e:
        logger.error(u"错误: %s" % str(e))
        import traceback
        traceback.print_exc()
        sys.exit(1)


if __name__ == "__main__":
    main()